{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: 流程控制和参数共享\n",
    "--------------------------------------\n",
    "\n",
    "为了展示PyTorch的动态图的能力，我们这里会实现一个很奇怪模型：这个全连接的网络的隐层个数是个1到4之间的随机数，\n",
    "而且这些网络层的参数是共享的。\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 667.2325439453125\n",
      "1 693.5020141601562\n",
      "2 661.39453125\n",
      "3 657.8023071289062\n",
      "4 629.8822021484375\n",
      "5 519.1797485351562\n",
      "6 653.5802612304688\n",
      "7 642.0776977539062\n",
      "8 637.75048828125\n",
      "9 584.5928344726562\n",
      "10 627.3572998046875\n",
      "11 643.9644775390625\n",
      "12 556.0167236328125\n",
      "13 638.9312744140625\n",
      "14 325.15899658203125\n",
      "15 632.3028564453125\n",
      "16 590.1881713867188\n",
      "17 501.169677734375\n",
      "18 483.6795959472656\n",
      "19 609.3463745117188\n",
      "20 438.00732421875\n",
      "21 210.1587677001953\n",
      "22 188.89439392089844\n",
      "23 366.6983642578125\n",
      "24 341.95196533203125\n",
      "25 560.5984497070312\n",
      "26 289.00738525390625\n",
      "27 450.74407958984375\n",
      "28 126.8094482421875\n",
      "29 491.33477783203125\n",
      "30 464.3042297363281\n",
      "31 431.2166442871094\n",
      "32 199.5053253173828\n",
      "33 371.03472900390625\n",
      "34 119.70623779296875\n",
      "35 191.9884490966797\n",
      "36 300.1385803222656\n",
      "37 273.79559326171875\n",
      "38 221.24205017089844\n",
      "39 233.88214111328125\n",
      "40 97.59255981445312\n",
      "41 191.1996307373047\n",
      "42 163.98350524902344\n",
      "43 80.37187194824219\n",
      "44 70.52588653564453\n",
      "45 122.30460357666016\n",
      "46 211.8968505859375\n",
      "47 104.52184295654297\n",
      "48 571.7366333007812\n",
      "49 161.8510284423828\n",
      "50 65.93932342529297\n",
      "51 463.068359375\n",
      "52 487.9062194824219\n",
      "53 319.4221496582031\n",
      "54 101.45733642578125\n",
      "55 285.63116455078125\n",
      "56 175.0655059814453\n",
      "57 132.19207763671875\n",
      "58 178.30162048339844\n",
      "59 370.3081359863281\n",
      "60 347.0126953125\n",
      "61 109.32127380371094\n",
      "62 149.53810119628906\n",
      "63 190.29037475585938\n",
      "64 125.62757873535156\n",
      "65 71.79491424560547\n",
      "66 62.97819137573242\n",
      "67 156.31350708007812\n",
      "68 37.14790725708008\n",
      "69 137.30043029785156\n",
      "70 27.00755500793457\n",
      "71 26.541860580444336\n",
      "72 57.17127990722656\n",
      "73 173.85784912109375\n",
      "74 99.54998016357422\n",
      "75 42.611488342285156\n",
      "76 75.09943389892578\n",
      "77 45.40507888793945\n",
      "78 118.73471069335938\n",
      "79 52.26136779785156\n",
      "80 39.81679916381836\n",
      "81 43.47416305541992\n",
      "82 29.002532958984375\n",
      "83 68.68782043457031\n",
      "84 34.37745666503906\n",
      "85 17.12519073486328\n",
      "86 29.176612854003906\n",
      "87 14.763790130615234\n",
      "88 13.174559593200684\n",
      "89 20.209497451782227\n",
      "90 59.81450653076172\n",
      "91 23.44327163696289\n",
      "92 20.816465377807617\n",
      "93 28.21196174621582\n",
      "94 18.710844039916992\n",
      "95 17.401988983154297\n",
      "96 48.45523452758789\n",
      "97 36.2074089050293\n",
      "98 22.05654525756836\n",
      "99 42.30142593383789\n",
      "100 22.608060836791992\n",
      "101 12.607400894165039\n",
      "102 41.6210823059082\n",
      "103 8.878849029541016\n",
      "104 15.206116676330566\n",
      "105 8.46290397644043\n",
      "106 19.30026626586914\n",
      "107 28.509586334228516\n",
      "108 17.39533042907715\n",
      "109 8.761665344238281\n",
      "110 15.525398254394531\n",
      "111 14.998927116394043\n",
      "112 24.602731704711914\n",
      "113 23.10215950012207\n",
      "114 15.447930335998535\n",
      "115 31.217288970947266\n",
      "116 25.0269718170166\n",
      "117 16.598876953125\n",
      "118 6.9173150062561035\n",
      "119 19.822355270385742\n",
      "120 15.722079277038574\n",
      "121 11.904041290283203\n",
      "122 15.876371383666992\n",
      "123 14.551761627197266\n",
      "124 11.999313354492188\n",
      "125 9.2424955368042\n",
      "126 18.692880630493164\n",
      "127 9.681963920593262\n",
      "128 7.575268745422363\n",
      "129 7.668093204498291\n",
      "130 17.841964721679688\n",
      "131 9.376633644104004\n",
      "132 4.73403263092041\n",
      "133 5.3814287185668945\n",
      "134 24.549375534057617\n",
      "135 15.65872573852539\n",
      "136 8.985286712646484\n",
      "137 4.199887752532959\n",
      "138 11.486433029174805\n",
      "139 23.929683685302734\n",
      "140 14.30218505859375\n",
      "141 5.149080276489258\n",
      "142 5.517061710357666\n",
      "143 10.568133354187012\n",
      "144 9.775389671325684\n",
      "145 6.29816198348999\n",
      "146 2.930241107940674\n",
      "147 6.935487747192383\n",
      "148 4.015586853027344\n",
      "149 9.535303115844727\n",
      "150 12.043564796447754\n",
      "151 12.112950325012207\n",
      "152 3.5908050537109375\n",
      "153 3.3790488243103027\n",
      "154 2.924522876739502\n",
      "155 2.3655924797058105\n",
      "156 3.1383111476898193\n",
      "157 4.442627906799316\n",
      "158 2.1676206588745117\n",
      "159 3.2454748153686523\n",
      "160 2.831533193588257\n",
      "161 9.970426559448242\n",
      "162 2.111205577850342\n",
      "163 6.009185314178467\n",
      "164 4.237937927246094\n",
      "165 3.945507287979126\n",
      "166 3.402341842651367\n",
      "167 2.3819901943206787\n",
      "168 1.748885154724121\n",
      "169 1.7254043817520142\n",
      "170 1.8824963569641113\n",
      "171 4.157584190368652\n",
      "172 3.428701400756836\n",
      "173 1.9711617231369019\n",
      "174 1.3451881408691406\n",
      "175 1.2320210933685303\n",
      "176 1.3237261772155762\n",
      "177 1.4217673540115356\n",
      "178 17.91507339477539\n",
      "179 7.977447032928467\n",
      "180 5.676833629608154\n",
      "181 7.523515701293945\n",
      "182 7.124314308166504\n",
      "183 5.291311740875244\n",
      "184 1.4607328176498413\n",
      "185 1.0916026830673218\n",
      "186 7.49027156829834\n",
      "187 2.122056484222412\n",
      "188 2.60264253616333\n",
      "189 5.601791858673096\n",
      "190 4.466485977172852\n",
      "191 2.616624355316162\n",
      "192 0.6813597083091736\n",
      "193 1.8410910367965698\n",
      "194 14.971527099609375\n",
      "195 5.942257404327393\n",
      "196 5.299971103668213\n",
      "197 4.794283866882324\n",
      "198 12.442940711975098\n",
      "199 7.087371349334717\n",
      "200 2.8380556106567383\n",
      "201 10.474052429199219\n",
      "202 4.228457927703857\n",
      "203 6.756955146789551\n",
      "204 2.142569065093994\n",
      "205 1.6598403453826904\n",
      "206 10.145039558410645\n",
      "207 0.6874836683273315\n",
      "208 6.034623622894287\n",
      "209 0.7753431797027588\n",
      "210 4.350340366363525\n",
      "211 0.8017674088478088\n",
      "212 4.833273887634277\n",
      "213 3.561028242111206\n",
      "214 4.754598617553711\n",
      "215 2.5591421127319336\n",
      "216 4.52529239654541\n",
      "217 5.288763046264648\n",
      "218 3.662212610244751\n",
      "219 2.0284667015075684\n",
      "220 6.508315086364746\n",
      "221 1.7959401607513428\n",
      "222 1.7432332038879395\n",
      "223 13.449140548706055\n",
      "224 2.1552953720092773\n",
      "225 1.721434235572815\n",
      "226 13.682909965515137\n",
      "227 0.9161562919616699\n",
      "228 4.443539142608643\n",
      "229 2.367795467376709\n",
      "230 0.8396650552749634\n",
      "231 5.2693071365356445\n",
      "232 0.5669422149658203\n",
      "233 0.6477370858192444\n",
      "234 3.3678839206695557\n",
      "235 2.050971031188965\n",
      "236 2.560544013977051\n",
      "237 1.2978224754333496\n",
      "238 3.4147942066192627\n",
      "239 1.7696864604949951\n",
      "240 1.3898844718933105\n",
      "241 1.1043651103973389\n",
      "242 1.1379106044769287\n",
      "243 3.1859548091888428\n",
      "244 0.9203171133995056\n",
      "245 1.2108144760131836\n",
      "246 1.1900759935379028\n",
      "247 1.0082497596740723\n",
      "248 0.793339192867279\n",
      "249 1.9532742500305176\n",
      "250 1.829748272895813\n",
      "251 0.9091793894767761\n",
      "252 0.8845037221908569\n",
      "253 2.2969541549682617\n",
      "254 0.5102388262748718\n",
      "255 0.43788453936576843\n",
      "256 0.3319489657878876\n",
      "257 1.9628714323043823\n",
      "258 1.0245307683944702\n",
      "259 0.9194345474243164\n",
      "260 1.8622878789901733\n",
      "261 0.3721359670162201\n",
      "262 0.3339320421218872\n",
      "263 2.55730938911438\n",
      "264 0.22296123206615448\n",
      "265 1.9461785554885864\n",
      "266 1.933531403541565\n",
      "267 0.2999608516693115\n",
      "268 0.9428123831748962\n",
      "269 1.1071789264678955\n",
      "270 2.809502363204956\n",
      "271 0.9429057240486145\n",
      "272 1.1392912864685059\n",
      "273 4.691129207611084\n",
      "274 1.5500035285949707\n",
      "275 2.650231122970581\n",
      "276 1.1087113618850708\n",
      "277 0.42377758026123047\n",
      "278 2.2388298511505127\n",
      "279 0.2547728717327118\n",
      "280 0.3690966069698334\n",
      "281 2.0954177379608154\n",
      "282 5.187933444976807\n",
      "283 2.2382936477661133\n",
      "284 0.6504868865013123\n",
      "285 4.476163864135742\n",
      "286 10.957630157470703\n",
      "287 7.277346611022949\n",
      "288 1.809762954711914\n",
      "289 16.17805290222168\n",
      "290 28.797801971435547\n",
      "291 0.9649246335029602\n",
      "292 23.934982299804688\n",
      "293 8.78559684753418\n",
      "294 1.503462553024292\n",
      "295 1.5533673763275146\n",
      "296 14.868090629577637\n",
      "297 7.615894794464111\n",
      "298 2.890125274658203\n",
      "299 10.367894172668457\n",
      "300 5.614292144775391\n",
      "301 0.985356330871582\n",
      "302 3.437028408050537\n",
      "303 12.845317840576172\n",
      "304 8.236506462097168\n",
      "305 1.9228476285934448\n",
      "306 2.557753086090088\n",
      "307 5.330724716186523\n",
      "308 12.52892017364502\n",
      "309 6.659863471984863\n",
      "310 3.2179858684539795\n",
      "311 7.979720115661621\n",
      "312 5.386771202087402\n",
      "313 1.2037687301635742\n",
      "314 3.391932964324951\n",
      "315 3.1241238117218018\n",
      "316 1.3181439638137817\n",
      "317 4.1960015296936035\n",
      "318 1.7400377988815308\n",
      "319 2.8874032497406006\n",
      "320 3.2124111652374268\n",
      "321 3.446425676345825\n",
      "322 1.146915316581726\n",
      "323 1.0745322704315186\n",
      "324 1.6437691450119019\n",
      "325 10.929742813110352\n",
      "326 2.0991952419281006\n",
      "327 1.6426520347595215\n",
      "328 3.636678695678711\n",
      "329 1.236626386642456\n",
      "330 1.4903663396835327\n",
      "331 1.1978909969329834\n",
      "332 1.352992057800293\n",
      "333 0.5205107927322388\n",
      "334 2.1647965908050537\n",
      "335 0.45925650000572205\n",
      "336 0.3929164707660675\n",
      "337 1.132492184638977\n",
      "338 0.28302329778671265\n",
      "339 1.0833364725112915\n",
      "340 0.2356039136648178\n",
      "341 2.398688316345215\n",
      "342 1.5576071739196777\n",
      "343 0.9702108502388\n",
      "344 1.247039556503296\n",
      "345 0.9263338446617126\n",
      "346 0.8984577059745789\n",
      "347 0.8224625587463379\n",
      "348 0.9381312131881714\n",
      "349 0.391269326210022\n",
      "350 1.174603819847107\n",
      "351 0.8499971032142639\n",
      "352 1.0256778001785278\n",
      "353 0.866655170917511\n",
      "354 0.5769461989402771\n",
      "355 0.7453818917274475\n",
      "356 0.7139766216278076\n",
      "357 0.7274928689002991\n",
      "358 0.46619340777397156\n",
      "359 0.34181085228919983\n",
      "360 1.2072926759719849\n",
      "361 0.7457610368728638\n",
      "362 0.6428603529930115\n",
      "363 0.14571276307106018\n",
      "364 1.2161239385604858\n",
      "365 0.8880175352096558\n",
      "366 0.779762864112854\n",
      "367 0.8882636427879333\n",
      "368 1.174759864807129\n",
      "369 0.7704432010650635\n",
      "370 0.3765392005443573\n",
      "371 0.7834899425506592\n",
      "372 0.6943578124046326\n",
      "373 0.9029036164283752\n",
      "374 0.5961515307426453\n",
      "375 0.7838281989097595\n",
      "376 0.3206554353237152\n",
      "377 1.218695044517517\n",
      "378 0.5778688192367554\n",
      "379 0.41854017972946167\n",
      "380 0.4969583749771118\n",
      "381 0.872016429901123\n",
      "382 0.37201830744743347\n",
      "383 0.15446695685386658\n",
      "384 0.4800300598144531\n",
      "385 0.7406393885612488\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "386 0.371745765209198\n",
      "387 0.49127039313316345\n",
      "388 0.5159963965415955\n",
      "389 0.4905443489551544\n",
      "390 0.3951423168182373\n",
      "391 0.5412530899047852\n",
      "392 0.4557349681854248\n",
      "393 0.44451403617858887\n",
      "394 0.39588019251823425\n",
      "395 0.5645430088043213\n",
      "396 0.2031405121088028\n",
      "397 0.19988282024860382\n",
      "398 0.16355444490909576\n",
      "399 0.1183689758181572\n",
      "400 0.08736192435026169\n",
      "401 0.07636848092079163\n",
      "402 0.5534483194351196\n",
      "403 0.4135391116142273\n",
      "404 0.410937637090683\n",
      "405 0.3865385353565216\n",
      "406 0.2663421928882599\n",
      "407 0.25469735264778137\n",
      "408 0.3201303780078888\n",
      "409 0.2853842079639435\n",
      "410 0.493245393037796\n",
      "411 0.1930563747882843\n",
      "412 0.16763263940811157\n",
      "413 1.0437934398651123\n",
      "414 0.11094801127910614\n",
      "415 0.3258151710033417\n",
      "416 0.11869552731513977\n",
      "417 0.17231100797653198\n",
      "418 0.6704407930374146\n",
      "419 0.7999902963638306\n",
      "420 0.12791769206523895\n",
      "421 0.4975522458553314\n",
      "422 0.6035558581352234\n",
      "423 0.4231157600879669\n",
      "424 0.40610167384147644\n",
      "425 0.463609904050827\n",
      "426 0.35453495383262634\n",
      "427 0.46370208263397217\n",
      "428 0.25054484605789185\n",
      "429 0.3317411243915558\n",
      "430 0.2783428132534027\n",
      "431 0.2581646144390106\n",
      "432 0.6921141147613525\n",
      "433 0.26949867606163025\n",
      "434 0.5620135068893433\n",
      "435 0.29463109374046326\n",
      "436 0.42994067072868347\n",
      "437 0.4843127429485321\n",
      "438 0.07578667253255844\n",
      "439 0.35023033618927\n",
      "440 0.4368148148059845\n",
      "441 0.5237947702407837\n",
      "442 0.38186681270599365\n",
      "443 0.09315255284309387\n",
      "444 0.0909242257475853\n",
      "445 0.47544121742248535\n",
      "446 0.06084638833999634\n",
      "447 0.39476877450942993\n",
      "448 0.38862094283103943\n",
      "449 0.04038674011826515\n",
      "450 0.38436561822891235\n",
      "451 0.3644045889377594\n",
      "452 0.32223010063171387\n",
      "453 0.2744787037372589\n",
      "454 0.45874089002609253\n",
      "455 0.46278512477874756\n",
      "456 0.1772507131099701\n",
      "457 0.4023905098438263\n",
      "458 0.16497240960597992\n",
      "459 0.1567007303237915\n",
      "460 0.1812146008014679\n",
      "461 0.8114114999771118\n",
      "462 0.7517392635345459\n",
      "463 0.28329911828041077\n",
      "464 0.564882218837738\n",
      "465 0.24391233921051025\n",
      "466 0.08021796494722366\n",
      "467 0.4851107597351074\n",
      "468 0.37306854128837585\n",
      "469 0.5060554146766663\n",
      "470 0.32402652502059937\n",
      "471 0.28989607095718384\n",
      "472 0.06614425778388977\n",
      "473 0.267289936542511\n",
      "474 0.24397625029087067\n",
      "475 0.3807356357574463\n",
      "476 0.08045618236064911\n",
      "477 0.22255682945251465\n",
      "478 0.16679824888706207\n",
      "479 0.6258715391159058\n",
      "480 0.6296910047531128\n",
      "481 0.058420997112989426\n",
      "482 0.4947265386581421\n",
      "483 0.3581884801387787\n",
      "484 0.33985966444015503\n",
      "485 0.3532368242740631\n",
      "486 0.2507281005382538\n",
      "487 0.5126703977584839\n",
      "488 0.19880236685276031\n",
      "489 0.5666860938072205\n",
      "490 0.1727060228586197\n",
      "491 0.5101724863052368\n",
      "492 0.306907057762146\n",
      "493 0.13281625509262085\n",
      "494 0.27265775203704834\n",
      "495 0.0890098288655281\n",
      "496 0.3478958010673523\n",
      "497 0.30498558282852173\n",
      "498 0.45869940519332886\n",
      "499 0.25490856170654297\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import torch\n",
    "\n",
    "\n",
    "class DynamicNet(torch.nn.Module):\n",
    "    def __init__(self, D_in, H, D_out):\n",
    "        \"\"\"\n",
    "        构造3个nn.Linear实例。\n",
    "        \"\"\"\n",
    "        super(DynamicNet, self).__init__()\n",
    "        self.input_linear = torch.nn.Linear(D_in, H)\n",
    "        self.middle_linear = torch.nn.Linear(H, H)\n",
    "        self.output_linear = torch.nn.Linear(H, D_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        输入和输出层是固定的，但是中间层的个数是随机的(0,1,2)，并且中间层的参数是共享的。\n",
    "        \n",
    "        因为每次计算的计算图是动态(实时)构造的，所以我们可以使用普通的Python流程控制代码比如for循环\n",
    "        来实现。读者可以尝试一下怎么用TensorFlow来实现。另外一点就是一个Module可以多次使用，这样就\n",
    "        可以实现参数共享。\n",
    "        \"\"\"\n",
    "        h_relu = self.input_linear(x).clamp(min=0)\n",
    "        for _ in range(random.randint(0, 3)):\n",
    "            h_relu = self.middle_linear(h_relu).clamp(min=0)\n",
    "        y_pred = self.output_linear(h_relu)\n",
    "        return y_pred\n",
    "\n",
    " \n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    " \n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    " \n",
    "model = DynamicNet(D_in, H, D_out)\n",
    " \n",
    "criterion = torch.nn.MSELoss(reduction='sum')\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)\n",
    "for t in range(500): \n",
    "    y_pred = model(x)\n",
    " \n",
    "    loss = criterion(y_pred, y)\n",
    "    print(t, loss.item())\n",
    " \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.6-env",
   "language": "python",
   "name": "py3.6-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
